import os
import json
from random import randint
from tqdm import tqdm
import pickle
import joblib
import random
import time

sp_char=['!','"','#','$','&','(',')','*','+',',','-','.','。','的','，','）','（','：','”','；','：','、']

def gen_all():
    path='/media/wsl/SB@data/OCR/SRNet-Datagen/corpus/new2016zh/news2016zh_train.json'
    f=open(path,encoding='utf8',mode='r')
    f1=open('./news_text.txt',mode='w',encoding='utf8')
    num=0
    for line in tqdm(f.readlines()):
        data = json.loads(line)
        content=data['content']
        content=content.strip()
        index=0
        while(index<len(content)):
            tl = randint(6, 15)
            if index + tl < len(content):
                f1.write('%s\n' % content[index:index + tl])
                index+=tl
            else:
                f1.write('%s\n' % content[index:-1])
                break
        if index<len(content):
            f1.write('%s\n' % content[index:-1])
        num+=1
        if num>30000:
            break
    f.close()
    f1.close()
def gen_balance():
    path = '/media/wsl/SB@data/OCR/SRNet-Datagen/corpus/new2016zh/news2016zh_train.json'
    f = open(path, encoding='utf8', mode='r')
    f1 = open('./news_text_balance.txt', mode='w', encoding='utf8')
    num = 0
    dicts={}
    total_char=0
    rate=0.001
    temp_res=''
    for line in tqdm(f.readlines()):
        # if randint(0,10)>0:
        #     continue
        data = json.loads(line)
        content = data['content']
        content = content.strip()
        # if randint(0,10)>0:
        #     for i in range(len(sp_char)):
        #         content=content.replace(sp_char[i],'')
        index = 0
        while (index < len(content)):
            tl = randint(6, 15)
            if index + tl < len(content):
                cur_content=content[index:index + tl]
                write_content=''
                if num<100000:
                    for j in range(len(cur_content)):
                        if not cur_content[j] in dicts.keys():
                            dicts[cur_content[j]]=1
                        else:
                            dicts[cur_content[j]] += 1
                        total_char+=1
                    write_content=cur_content
                else:
                    for j in range(len(cur_content)):
                        if not cur_content[j] in dicts.keys():
                            dicts[cur_content[j]]=1
                            write_content+=cur_content[j]
                            total_char+=1
                        else:
                            if dicts[cur_content[j]]/total_char<rate:
                                dicts[cur_content[j]] += 1
                                write_content += cur_content[j]
                                total_char+=1
                if len(write_content)>5:
                    f1.write('%s\n' % write_content)
                else:
                    temp_res+=write_content
                    if len(temp_res)>5:
                        f1.write('%s\n' % temp_res)
                        temp_res=''
                index += tl
            else:
                cur_content = content[index:-1]
                write_content = ''
                if num < 100000:
                    for j in range(len(cur_content)):
                        if not cur_content[j] in dicts.keys():
                            dicts[cur_content[j]] = 1
                        else:
                            dicts[cur_content[j]] += 1
                        total_char += 1
                    write_content = cur_content
                else:
                    for j in range(len(cur_content)):
                        if not cur_content[j] in dicts.keys():
                            dicts[cur_content[j]] = 1
                            write_content += cur_content[j]
                            total_char += 1
                        else:
                            if dicts[cur_content[j]] / total_char < rate:
                                dicts[cur_content[j]] += 1
                                write_content += cur_content[j]
                                total_char+=1
                if len(write_content)>5:
                    f1.write('%s\n' % write_content)
                else:
                    temp_res+=write_content
                    if len(temp_res)>5:
                        f1.write('%s\n' % temp_res)
                        temp_res=''
                break
            num += 1
            if num > 2000000:
                break
    print('总共%d条语料'%num)
    f.close()
    f1.close()
def gen_number():
    cls='0123456789QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm()-+:*%<>/[]'
    #cls='0123456789QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm'
    cls=[c for c in cls]
    f=open('./random.txt',mode='w')
    for i in range(1000000):
        if randint(0,10)<5:
            string=get_time()
        else:
            length=randint(10,20)
            string=random.choices(cls,k=length)
            string=''.join(string)
        f.write(string+'\n')
    f.close()

def get_time():
    a1 = (1000, 1, 1, 0, 0, 0, 0, 0, 0)  # 设置开始日期时间元组（1976-01-01 00：00：00）
    a2 = (3000, 12, 31, 23, 59, 59, 0, 0, 0)  # 设置结束日期时间元组（1990-12-31 23：59：59）
    
    start = time.mktime(a1)  # 生成开始时间戳
    end = time.mktime(a2)  # 生成结束时间戳
    t = random.randint(start, end)  # 在开始和结束时间戳中随机取出一个
    date_touple = time.localtime(t)  # 将时间戳生成时间元组
    date_ymd=time.strftime("%Y/%m/%d", date_touple)
    date_mdy = time.strftime("%m/%d/%Y", date_touple)
    date_all = time.strftime("%Y/%m/%d/%H/%M/%S", date_touple)  # 将时间元组转成格式化字符串（1976-05-21）
    rand_v=randint(0,10)
    if rand_v<3:
        return date_ymd
    elif rand_v<6:
        return date_mdy
    else:
        return date_all

def get_corpus():
    
    cls = '0123456789QWERTYUIOPASDFGHJKLZXCVBNM()-+:*%<>/[]'
    cls = [c for c in cls]
    f = open('./random.txt', mode='w')
    for i in range(1000000):
        if randint(0, 10) < 6:
            string =get_time()
        else:
            length = randint(5, 15)
            string = random.choices(cls, k=length)
            string = ''.join(string)
        f.write(string + '\n')
    f.close()
def get_hoke_time():
    a1 = (2000, 1, 1, 0, 0, 0, 0, 0, 0)  # 设置开始日期时间元组（1976-01-01 00：00：00）
    a2 = (2100, 12, 31, 23, 59, 59, 0, 0, 0)  # 设置结束日期时间元组（1990-12-31 23：59：59）

    start = time.mktime(a1)  # 生成开始时间戳
    end = time.mktime(a2)  # 生成结束时间戳
    t = random.randint(start, end)  # 在开始和结束时间戳中随机取出一个
    date_touple = time.localtime(t)  # 将时间戳生成时间元组
    if randint(0, 10) < 5:
        d1=time.strftime("%Y%m%d-", date_touple)
        t = random.randint(start, end)  # 在开始和结束时间戳中随机取出一个
        date_touple = time.localtime(t)  # 将时间戳生成时间元组
        d2=time.strftime("%Y%m%d", date_touple)
        date=d1+d2

    else:
        date = time.strftime("%Y%m%dP1%H:%M(Z)", date_touple)  # 将时间元组转成格式化字符串（1976-05-21）
    return ''.join(date)

def gen_haoke():
    cc='0123456789QWERTYUPASDFGHJKLZXCVBNMqwertyupasdfghjklzxcvbnm'
    f=open('./haoke.txt',mode='w')
    for i in range(100000):

        string=get_hoke_time()
        f.write(string+'\n')
    f.close()
if __name__=='__main__':
    gen_haoke()
    #get_corpus()
    #print(get_time())

    #print(sp_char)
    #gen_balance()
    #gen_number()
    # f = open('./hoke.txt', mode='w')
    # for i in range(1000000):
    #
    #     string = get_hoke_time()
    #
    #     f.write(string + '\n')
    # f.close()